'''
Created on Feb 3, 2011

@author: Veerbhan
'''

from modules.classificationAPI import *
import numpy
import re
__description__ = "API for DataTable Structure"
import sys
from sets import Set
import string
import re
import csv
import gzip
import math

__all__ = ['DataTable']
class DataTable(dict):
    """ Data Table Class
    """
    
    def goodCheck(self,diffValue):
        if diffValue < 0.2:
            return 1.0
        else:
            return 0
        
        
    def formatFloat(self,data):
        if type(data) == type([]) or type(data) == type(()):
            ret = []
            for n in data:
                ret.append(self.formatFloat(n))
            return ret
#        elif type(data) == type(123):
#            return str(data)
        else:
            data = '%.12g' % float(data)
            try:
                intData = int(data)
                return str(intData)
            except:
                n = float(data)
                floatData = "%.1e" % n if n and (math.log10(abs(n)) < -3 or math.log10(abs(n)) > 3) else ("%.3f" % n).rstrip("0")
#            if math.fabs(floatData) < 1000.0 and math.fabs(floatData) > 0.01:
#                floatData = '%.3f' % floatData
#            elif floatData == 0:
#                floatData = '0'
#            else:
#                floatData = '%.1e' % floatData
                return floatData
        
    def isMonotonic(self,L,spec='',threshold=8e-9):
        data = list(L)
        #current under threshold is unreliable, pin down to 8e-9
        for i,n in enumerate(data):
            if n < threshold:
                data[i]=threshold
        if spec.upper() == 'LOG':
            data = [math.log(y) for y in data]
            threshold = math.fabs(math.log(threshold))

        return all(x>=y for x, y in zip(data, data[1:]) if math.fabs(x-y) > threshold) \
             or all(x<=y for x, y in zip(data, data[1:]) if math.fabs(x-y) > threshold)
#        if all(x>=y for x, y in zip(data, data[1:])) or all(x<=y for x, y in zip(data, data[1:])):
#            return True
#        else:
#            print "y:",data
#            return False
        
    def findMedian(self,data):
        if type(data)==type([]):
            if len(data) > 1:
                data2sort = list(data)
                data2sort.sort()
                if (len(data2sort)%2):
                    medianValue = data2sort[len(data2sort)/2]                    
                else:                       
                    medianValue = (data2sort[len(data2sort)/2-1] + data2sort[len(data2sort)/2])/2.0
                return medianValue
            else:
                return data[0]
        elif type(data)==type({}):
            ret = []
            for k,v in data.items():
                if type(v)==type([]):
                    ret.extend(v)
                elif type(v) == type(123) or type(v) == type(1.0):
                    ret.append(v)
                else:
                    raise "unknown type: ",type(v)
            return self.findMedian(ret)
        else:
            return data
    
    def stdevCalcu(self,data):
        if type(data)==type([]):
            return numpy.std(data)
        elif type(data)==type({}):
            ret = []
            for k,v in data.items():
                if type(v)==type([]):
                    ret.extend(v)
                elif type(v) == type(123) or type(v) == type(1.0):
                    ret.append(v)
                else:
                    raise "unknown type: ",type(v)
            return self.stdevCalcu(ret)
        else:
            raise "unsuported type: ",type(data)
    
    def convert2curve(self,data):
        """
        convert the values in data into a list of x-axis value(in even position)
        and a list of y-axis value(in odd position), for example:
        data='1.0,3.13,2.0,0.6,3.0,0.01'
        we got 
        xValues = [1.0,2.0,3.0]
        yValues = [3.13,0.6,0.01]
        """
        xValues = []
        yValues = []
        #collect all the value for voltage
        vMap = {}
        tmp_data = []
        for d in data:
            if type(d) == type([]):
                tmp_data.extend(d)
            else:
                tmp_data.append(d)
        for i in xrange(len(tmp_data)-1):
            if i % 2 == 0:
                try:
                    vMap[tmp_data[i]].append(float(tmp_data[i+1]))
                except:
                    vMap[tmp_data[i]] = [float(tmp_data[i+1]),]
        sorted_vMap = sorted(vMap.iteritems(), key=lambda vMap:vMap[0])
    
        for p in sorted_vMap:
            xValues.append(p[0])
            yValues.append(self.findMedian(p[1]))
        return xValues,yValues
             
    #def __setattr__(self,key,value):
    #    self[key]=value
    def __setattr__(self, item, value):
        """Maps attributes to values.
        Only if we are initialised
        """
        if not self.__dict__.has_key('_attrExample__initialised'):  # this test allows attributes to be set in the __init__ method
            return dict.__setattr__(self, item, value)
        elif self.__dict__.has_key(item):       # any normal attributes are handled normally
            dict.__setattr__(self, item, value)
        else:
            self.__setitem__(item, value)
    
    def __getattr__(self, key):
        if key == 'row':
            key='data'
        return dict.__getitem__(self,key)

    def getDictForRow(self,row):
        ret = {}
        for i,n in enumerate(self['columns']):
            ret[n] = row[i]
        return ret

    def getItem(self,rowIndex):
        r = self['data'][rowIndex]
        ret = {}
        for i,n in enumerate(self['columns']):
            ret[n] = r[i]
        return ret
    
    def __init__(self,columns=None,data=None,csv=None):
        if columns == None:
            self['columns'] = []
        else:
            if type(columns) == type('ABC'):
                columns = columns.split(',')
            else:
                if type(columns) == type([1,2,3]) or type(columns) == type((1,2,3)):
                    pass
                else:
                    raise TypeError
            self['columns'] = columns[:]
            
        if data == None:
            self['data'] = []
        else:
            self['data'] = data
            
        if csv != None:
            self.readCSV(csv)
        
        self.attributes={}
        self.indexColumns={}
        #self.hash = dict()
        self.dataMask = dataFilter() #setup empty mask

    def setIndexColumns(self,indexCols):
        self.indexColumns = indexCols

    def isIndexColumn(self,colName):
        try:
            return self.indexColumns[colName]
        except KeyError:
            return -1

    def setAttributes(self,k,v=None):
        if type(k) == type({}):
            for a,b in k.items():
                self.attributes[a] = b
        else:
            self.attributes[k] = v
            

    def readTabSeparatedFile(self,tsvfile,readColumns=None):
        read = open(tsvfile,'r')
        if readColumns == None:
            readColumns = []
        else:
            readColumns = readColumns.split(',')
        firstDone = False
        self['data'] = []
        print "Reading CSV file:",tsvfile
        numcols=0
        #self['columns'] = ['pattern', 'neighbor', 'x', 'y', 'max:PARETO_ALL']
        allowedCols = []
        for line in read:
            line = line.rstrip("\n")
            line = line.rstrip("\r")
            #if line[-1] == '\n':
                
            #line.replace('"',"'")
            oldline = line
            line = ''
            
            #rege = re.compile("[\d]+\.?[\d]+")
            #reges = re.compile("[\d]")
            for itemIndex,n in enumerate(oldline.split('\t')):
                
                if len(allowedCols) > 0:
                    if itemIndex in allowedCols:
                        pass
                    else:
                        continue
                
                n = string.rstrip(string.lstrip(n))
                if n == '---' or n=='NA':
                    line = line + "None" + ','
                else:
                    try:
                        x = float(n)
                        line = line + n + " , "
                    except:
                        line = line + "'" + n + "' ,"
                
            if oldline == '':
                continue
            
            if line[0] != '[':
                line = '[' + line + ']'
            #print "newline:",line 
            if firstDone:
                #print "<<<","self['data'].append(" + line + ")",">>>"
                eval("self['data'].append(" + line + ")")
                if len(self['data'][-1]) < numcols:
                    print "Error: data is line is < number of columns.\n",line
                    raise
            else:
                exec("self['columns'] = " + line)
                newcol = []
                if len(readColumns) > 0:
                    for indx,allc in enumerate(self['columns']):
                        if allc in readColumns:
                            newcol.append(allc)
                            allowedCols.append(indx)
                else:
                    print "Reading all columns"
                    for indx,allc in enumerate(self['columns']):
                        newcol.append(allc)
                        allowedCols.append(indx)
                
                self['columns'] = newcol
                     
                
                numcols = len(self.columns)
                print "Reading... number of columns:",numcols
                firstDone = True
        
        
        print "Finished reading Tab separated file (",len(self.data),"rows )."
        assert len(self['columns']) > 0        


    

    def readGroupedCSV(self,csvfile): 
        breakp = re.compile('   ([0-9]+), ([A-Za-z_\.0-9]+): ')
        read = open(csvfile,'r')
        firstDone = False
        self['data'] = []
        extraColumn=None
        #self['columns'] = ['pattern', 'neighbor', 'x', 'y', 'max:PARETO_ALL']
        for lineno,line in enumerate(read):
            line = line.rstrip("\n")
            line = line.rstrip("\r")
            #if line[-1] == '\n':
                
            line.replace('"',"'")
            if firstDone:
                #print line
                matchbreakp = breakp.search(line)
                if matchbreakp:
                    if extraColumn:
                        assert extraColumn == matchbreakp.group(2)
                    else:
                        extraColumn = matchbreakp.group(2)
                        self['columns'].append(extraColumn)
                    #print matchbreakp.group(0)
                    #print matchbreakp.group(1)
                    #print matchbreakp.group(2)
                    lastColumnData = line[matchbreakp.end():]
                    line = line[:matchbreakp.start()]
                    #print line[matchbreakp.end():]
                else:
                    raise "Error reading GroupedCSV at line number ",lineno
            oldline = line
            line = ''
            #rege = re.compile("[\d]+\.?[\d]+")
            #reges = re.compile("[\d]")
            for n in oldline.split(','):
                n = string.rstrip(string.lstrip(n))
                if n=='':
                    continue
                if n == '---' or n=='NA':
                    line = line + "None" + ','
                else:
                    try:
                        x = float(n)
                        line = line + n + " , "
                    except:
                        line = line + '"' + n + '" ,'
                
            if line[0] != '[':
                line = '[' + line + ']'
            #print "newline:",line 
            if firstDone:
                rowdata = eval(line)
                rowdata.append(eval('['+lastColumnData+']'))
                self['data'].append(rowdata)
                               
                #eval("self['data'].append(" + line + ")")
            else:
                exec("self['columns'] = " + line)
                firstDone = True
        assert len(self['columns']) > 0
        
    
    
    def readCSV(self,csvfile,loadColumns=None,maxRows=1e+9,readFromString=None,nonePadding=None,includeALL=False,maxColNum=None): 
        
        if readFromString == None:
            if csvfile[-3:] == '.gz' or csvfile[-5:] == '.gzip':
                print "Reading Zip -> CSV file:",csvfile
                read = gzip.open(csvfile,'rb')
            else:
                print "Reading CSV file:",csvfile
                read = open(csvfile,'r')
        else:
            read = readFromString.split('\n')
            #print read
            
        firstDone = False
        self['data'] = []
        
        numcols=0
        if maxColNum == None:
            maxColNum = numcols
        lineno = 0
        mil=0
        #self['columns'] = ['pattern', 'neighbor', 'x', 'y', 'max:PARETO_ALL']
        for linenumber,line in enumerate(read):
            
            if linenumber > maxRows:
                break
            
            lineno = lineno + 1
            if lineno > 1e+5:
                mil=mil+1
                lineno = 0
                sys.stdout.write("["+str(mil)+"x100K]")
                sys.stdout.flush()
            
            orgline = line
            line = line.rstrip("\n\r")
            #if line[-1] == '\n':
                
            line.replace('"',"'")
            oldline = line
            line = []
            
            #rege = re.compile("[\d]+\.?[\d]+")
            #reges = re.compile("[\d]")
            for n in oldline.split(','):
                n = string.rstrip(string.lstrip(n))
                if n == '---' or n=='NA' or n == '' or n == '-':
                    line.append(None)
                    #line = line + "None" + ','
                else:
                    try:
                        x = float(n)
                        line.append(x)
                        #line = line + n + " , "
                    except:
                        line.append(n) #'"' + n + '"')
                        #line = line + '"' + n + '" ,'
                
            if oldline == '':
                continue
            
            #if line[0] != '[':
            #    line = '[' + line + ']'
            
            #print "newline:",line 
            if firstDone:
                if includeALL and len(line) > maxColNum:
                    lastElem = []
                    for n in line[maxColNum-1:]:
                        if n == '' or n == None or n == 'None':
                            continue
                        if type(n) == type('a'):                            
                            lastElem.extend([float(i) for i in n.strip("[]' \n").split()])
                        else:
                            lastElem.append(n)
                                
                    line = line[:maxColNum-1]
                    line.append(lastElem)
                        
                self['data'].append(line)
                #eval("self['data'].append(" + line + ")")
                if includeALL:
                    pass
                else:
                    if len(self['data'][-1]) < numcols:
                        if nonePadding != None:
                            if nonePadding:
                                numpad = numcols-len(self['data'][-1])
                                for nnn in xrange(numpad):
                                    self['data'][-1].append(None)
                        else:
                                
                            print "Error: data columns < number of expected columns - At line number:",linenumber,"\n",orgline
                            raise
            else:
                self['columns'] = line
                #exec("self['columns'] = " + line)
                
                numcols = len(self.columns)
                print "Number of columns:",numcols
                firstDone = True
        
        auto=1
        for i,n in enumerate(self['columns']):
            
            if n == None:
                self['columns'][i]= "NA_auto" + str(auto)
                auto = auto+1
                
        if auto > 1:
            print "Found ",auto-1,"columns with no names. They have been assigned names NA_auto*" 
        print "Finished reading CSV (",len(self.data),"rows )."
        assert len(self['columns']) > 0
           
    
    def __str__(self):
        ret = ''
           
        cindex = []
        ret = ret + 'Columns:   [' + ','.join([str(i) for i in self['columns']]) + ']\n\nAttributes: "' + str(self.attributes) + '\n\nData: \n'
        for i,n in enumerate(self['data']):
            if i > 10:
                ret = ret + '...\n...\n...\n<' + str(len(self.data)) + ' data rows>\n'
                break;
            fn = ''
            #for e in n:
            #    if type(e)==type(1.0):
            #        fn = fn + '%3.3f' % (e)
            #    else:
            #        fn = fn + str(e)
            #    fn = fn + ','
            
            ret = ret + '<' + str(i) + '>   [' + ','.join([('"' + i + '"' if type(i) == type('') else str(i)) for i in n]) + ']\n'
        return ret
    
    
    def __repr__(self):
        return self.__str__()
    
    
    
    def writeCSVExcel(self,csvfile=None,columns=[],filter=None,delimeter=','):
        if type(columns) != type([]):
            print "Incorrect argument! 'Columns' argument must be of type list!!"
            raise
        
        if csvfile:
            out = open(csvfile,'w')
        else:
            out = sys.stdout
        if len(columns) == 0:
            st = delimeter.join(self['columns'])
            #for j in self['columns']:
            #    st = st + j + ','
            print >> out,st 

            if filter:
                for n in filter.filter(self):
                    print >>out, n
            else:
                for n in self['data']:
                    st = ''
                    dele=''
                    for j in n:
                        if j == None:
                            st = st + dele + '-'
                        else:
                            st = st + dele + str(j)
                        dele = delimeter
                    print >>out,st

        else:
            st = delimeter.join(columns)
            #for j in self['columns']:
            #    st = st + j + ','
            print >> out,st 
            
            cindex = []
            #print >>out,columns
            for n in columns:
                cindex.append(self.getColumnIndex(n))
            if filter:
                for n in filter.filter(self):
                    print >>out,self.__filterRow(n,cindex)
            else:
                for n in self['data']:
                    st = ''
                    dele=''
                    for ii in cindex:
                        if ii >= len(n):
                            print "Error: Record too small:",n
                            print "Looking for ",ii
                            raise ValueError
                        j = n[ii]
                        if j == None:
                            st = st + dele + '-'
                        else:
                            st = st + dele + str(j)
                        dele = delimeter
                    print >>out,st
                    #print >>out,self.__filterRow(n,cindex)
        if csvfile:
            out.close()
        else:
            print >>out,''
        
    def writeCSV(self,csvfile=None,columns=[],filter=None):
        if csvfile:
            out = open(csvfile,'w')
        else:
            out = sys.stdout
        if len(columns) == 0:
            print >>out,','.join(self['columns'])
            if filter:
                for n in filter.filter(self):
                    print >>out, n
            else:
                for n in self['data']:
                    print >>out,','.join([str(i) for i in n])

        else:
            
            cindex = []
            print >>out,','.join(columns)
            for n in columns:
                cindex.append(self.getColumnIndex(n))
            if filter:
                for n in filter.filter(self):
                    print >>out,self.__filterRow(n,cindex)
            else:
                for n in self['data']:
                    print >>out,self.__filterRow(n,cindex)
                    
                
        
    def __filterRow(self,row,cindex):
        frow = []
        for n in cindex:
            frow.append(row[n])
        return frow    


    def __getColumnIndexNoException(self,columnName):
        index =0 
        if type('a') == type(columnName):
            for n in self['columns']:
                if n == columnName:
                    return index
                index = index + 1
            return -1
        else:
            assert type(columnName) == type([]) or type(columnName) == type(())
            ret = []
            for n in columnName:
                try:
                    i = self['columns'].index(n)
                    ret.append(i)
                except:
                    ret.append(-1)
            return tuple(ret)
    
    def hasColumn(self,columnName):
        if columnName in self['columns']:
            return True
        else:
            return False
        
    def getColumnIndex(self,columnName):
        
        index =0 
        if type('a') == type(columnName):
            for n in self['columns']:
                if n == columnName:
                    return index
                index = index + 1
                
            print "Error: Cannot find column: '"+columnName + "' in " + str(self['columns'])
            raise
        else:
            
            assert type(columnName) == type([]) or type(columnName) == type(())
            ret = []
            for n in columnName:
                try:
                    i = self['columns'].index(n)
                    ret.append(i)
                except:
                    raise "cannot find column: "+n + " in " + str(self['columns'])
            return tuple(ret)
    

    def getData(self,row,column):
        
        return self['data'][row][column]
    
    def setData(self,row,column,newData):
        self['data'][row][column] = newData
        
    def getColumnData(self,columnIndex,filter=None):
        if type(columnIndex) == type(''):
            columnIndex = self.getColumnIndex(columnIndex)
        ret = []
        #if classId:
        #    classIdColumn = self.getColumnIndex('Consolidated Class Id')
        if filter:
            for n in filter.filter(self):
                ret.append(n[columnIndex])
        else:
            for n in self['data']:
                ret.append(n[columnIndex])

        return ret

    def __setDataMask(self,datafilter):
        self.dataMask  = datafilter
    
    def __clearDataMask(self):
        self.dataMask = dataFilter()

    def getColumns(self,columnNameList,filter=None):
        ret = DataTable()
        ret['columns'] = columnNameList
        ret['data'] = []
        evalstr = '['
        for n in ret['columns']:
            evalstr = evalstr + 'n[' + str(self.getColumnIndex(n)) + '],'
        evalstr += ']'
        evalstr = "ret['data'].append(" + evalstr + ")"
        cccode = compile(evalstr,'<string>','eval')
        if filter:
            for n in filter.filter(self):
                eval(cccode)
        else:
            for n in self['data']:
                #print n,evalstr
                eval(cccode)
        return ret
    
    
    def union(self,dtable):  
        ret = DataTable()

        
        columns = self['columns'][:]
        dtabletupstr = '['
        c1 = len(columns)
        
        for n in dtable['columns']:
            if n in columns:
                pass
            else:
                columns.append(n)
        
        c2= len(columns)
        
        data= []
        for n in self['data']:
            r = n[:]
            r.extend([None for n in xrange(c2-c1)])
            data.append(r)
        dtablelookup = ['None' for n in xrange(c2)]
        for nindex,n in enumerate(dtable['columns']):
            i = columns.index(n)
            dtablelookup[i] = 'n[' + str(nindex) + ']'
        dtablelookupstr = '[' + ','.join(dtablelookup) + ']'
        for n in dtable['data']:
            r = eval(dtablelookupstr)
            data.append(r)
            
        ret['columns'] = columns
        ret['data'] = data
        return ret
        
        
        
#            assert self['columns'] == dtable['columns']
#        ret['columns'] = dtable['columns']
#        ret['data'] = self['data'][:]
#        for n in dtable['data']:
#            ret['data'].append(n)
#        return ret
    
    

        
    def innerproduct(self,dtable):
        ret = DataTable()
        ret['columns'] = self['columns'][:]
        
        for cindex,c in enumerate(dtable['columns']):
            ret['columns'].append(c)

        ret['data'] = []

        for n in self['data']:
            for m in dtable['data']:
                row = n[:]
                row.extend(m)
                ret['data'].append(row)
        
        return ret

    def join(self,dtable,pivotColumns,jointype='left',defaults=dict(),hashFunction=None):
        if type(pivotColumns) == type(''):
            pivotColumns = pivotColumns.split(',')
        
        
        if hashFunction == None:
            hashFunction = tuple
        
        defaultleftjoinvalues = dict()
        sindex = self.getColumnIndex(pivotColumns)
        dindex = dtable.getColumnIndex(pivotColumns)
        
        innerproductcolumns=None
        if innerproductcolumns:
            ipcolumns = dtable.getColumnIndex(innerproductcolumns)
        
        
        pivotkeys = {}
        ret = DataTable()
        ret['columns'] = self['columns'][:]
        appendst = '['
        appendEmpty = []
        for cindex,c in enumerate(dtable['columns']):
            if cindex in dindex:
                pass
            else:
                if c in self['columns']:
                    #if c in self['columns']:
                    #k = self.getColumnIndex(c)
                    print "Duplicate column:",c,"will be dropped during innerjoin"
                else:
                    appendst += 'n[' + str(cindex) + '],'
                    ret['columns'].append(c)
                    if defaults.has_key(c):
                        defaultleftjoinvalues[len(ret['columns'])-1] = defaults[c]
                        
                    if jointype == 'innerproduct':
                        if cindex in ipcolumns:
                            appendEmpty.append(None)
                        else:
                            appendEmpty.append('*')
                    else:
                        appendEmpty.append(None)
                     
        appendst += ']'
        appendstcode = compile('rowdata.extend('+appendst+')','<string>','exec')        
        for i,n in enumerate(dtable['data']):
            key = hashFunction([n[p] for p in dindex])
            if pivotkeys.has_key(key):
                pivotkeys[key].append(i)
            else:
                pivotkeys[key] = [i,]
                
        print "Table built."
        ret['data'] = []
        for j in self['data']:
            key = hashFunction([j[i] for i in sindex])
            if pivotkeys.has_key(key):
                #print "found key:",key,pivotkeys[key]
                for i in pivotkeys[key]:
                    rowdata = j[:]
                    n = dtable['data'][i]
                    exec(appendstcode)
                    #eval('rowdata.extend(' + appendst + ')')
                    ret['data'].append(rowdata)
            else:
                if jointype == 'inner':
                    pass
                elif jointype == 'innerstrict':
                    print "Error: Cannot find join keys for ",key," in target table during innerstrict join"
                    raise ValueError
                elif jointype == 'innerwarning':
                    print "Warning: Cannot find join keys for ",key," in target table during innerstrict join"
                    
                else:
                    if jointype == 'left' or jointype == 'innerproduct':
                        #print "key not found in dtable:",key
                        rowdata = j[:]
                        #n = dtable['data'][i]
                        rowdata.extend(appendEmpty)
                        for key,val in defaultleftjoinvalues.items():
                            rowdata[key] = val
                        ret['data'].append(rowdata)
                    else:
                        raise "jointype must be inner or left"
                                
        return ret

    
    
    
    
    def __findColumns(self,regularExpression):
        ret = []
        for n in self['columns']:
            if re.search(regularExpression,n):
               ret.append(n)
               
        return ret 
    
    def renameColumn(self,columnName,newName):
        if self.hasColumn(newName):
            print "Error: Column ",newName," already exists!"
            raise 
        
        try:
            cindex = self['columns'].index(columnName)
        except ValueError:
            print "Error: Cannot find",columnName,"in the list of columns."
            raise
        
        self['columns'][cindex]=newName
        
    def rowCount(self):
        return len(self['data'])
    
    def columnCount(self):
        return len(self['columns'])
    
    def replaceColumnFunction(self,columnName,function):
        cindex = self.getColumnIndex(columnName)
        for row in self['data']:
            d = row[cindex]
            processd = function(d)
            
            row[cindex] = processd
        
    def __computeClassVector(self,columnName,expressionVector):
        ccl=list(expressionVector)
        ccl = []
        for ex in expressionVector:
            cc = ex
            for n in sorted(self['columns'],key=lambda k: len(k), reverse=True):
                index = self.getColumnIndex(n)
                cc = cc.replace(n,'row['+str(index)+']')
            ccl.append(cc)
        self['columns'].append(columnName)
        for row in self['data']:
            rowval = 0
            try:
                for cc in ccl:
                    newval = eval(cc)
                    if newval == True:
                        newval =1
                    else:
                        if newval == False:
                            newval = 0
                        else:
                            print cc," must be a conditional expression"
                            raise
                    rowval = (rowval << 1) | newval
                    
            except:
                print "Column not found during ccvector ",expressionVector
                print self['columns']
                print cc
                print row
                raise
            row.append('C'+str(rowval))
                
                            


    def replaceColumn(self,columnName,expression,obj=None):
        cindex = self.getColumnIndex(columnName)
        cc = expression
        checkcc = []
        for n in sorted(self['columns'],key=lambda k: len(k), reverse=True):
            index = self.getColumnIndex(n)
            oldcc = cc
            cc = oldcc.replace(n,'row['+str(index)+']')
            if cc != oldcc:
                checkcc.append(' row[' + str(index) +'] != None ')
        
        checkexp = ' and '.join(checkcc)
        #print checkexp
        #self['columns'].extend(columnName)
        for row in self['data']:
            try:
                if eval(checkexp):
                    newval = eval(cc)
                else:
                    newval = None
            except:
                print "Column not found during replace ",expression
                print self['columns']
                print cc
                print row
                raise
            if newval == True:
                newval = 1
            else:
                if newval == False:
                    newval =0
            row[cindex] = newval

    

        #print cc
        
        #bitpos =0
        #for i in xrange(len(self.data['columns'])):
        #    if (self.data['columns'][i]).find('Filter:') >= 0:
        #        bitpos = bitpos + 1
    
        #filterColumnIndex = len(self.data['columns'])
        #self.data['columns'].append('Filter: ' + condition)
    def addDataIndexColumn(self,columnName):
        self['columns'].append(columnName)
        for i,n in enumerate(self['data']):
            n.append(i)
        return len(self['columns'])-1
    
    def addColumn(self,columnName,expressionStr,obj=None):
        self.computeColumn(columnName, expressionStr, obj)


    def computeColumn(self,columnName,expression,obj=None):
        if type('a') == type(columnName) and type('') == type(expression):
            pass
        else:
            raise "columnName & expressionStr must be a string"
        #    columnName = [columnName,]
        if self.hasColumn(columnName):
            print "Error: Column ",columnName," already exists!"
            raise
        
        cc = expression
        checkcc = []
        for n in sorted(self['columns'],key=lambda k: len(k), reverse=True):
            index = self.getColumnIndex(n)
            oldcc = cc
            cc = oldcc.replace(n,'row['+str(index)+']')
            if cc != oldcc:
                checkcc.append(' row[' + str(index) +'] != None ')
        if len(checkcc) == 0:
            checkexp = 'True'
        else:
            checkexp = ' and '.join(checkcc)
        #print checkexp,cc
        self['columns'].append(columnName)
        for row in self['data']:
            try:
                if eval(checkexp):
                    newval = eval(cc)
                else:
                    newval = None
            except:
                print "Column not found during ",expression
                print self['columns']
                print cc
                print row
                raise
            if newval == True:
                newval = 1
            else:
                if newval == False:
                    newval =0
            row.append(newval)
#            else:
#                row.extend(newval)        



       
        



    
    def computeColumnExecVersion(self,columnName,expression,obj=None):
        if type('a') == type(columnName) and type('') == type(expression):
            pass
        else:
            raise "columnName & expressionStr must be a string"
        #    columnName = [columnName,]
        
        
        cccode = compile(expression,'<string>','eval')
        
        nonecount =0
        for row in self['data']:
            try:
                newval = eval(cccode,self.getDictForRow(row))
            except NameError:
                print "Column not found during ",expression
                print self['columns']
                #print cc
                print row
                raise

            except TypeError:
                if nonecount == 0:
                    print "Warning: Cannot compute",expression,"for row",row
                nonecount = nonecount +1
                newval = None
            
            
            
            if newval == True:
                newval = 1
            else:
                if newval == False:
                    newval =0
            row.append(newval)
        
        self['columns'].append(columnName)
        if nonecount > 0:
            print "Warning: Further warnings suppresed:",nonecount
        
         
    
    def __getSortKeyComplex(self,keyOrder):
        lam = '('
        dele = ''
        for n in keyOrder:
            lam = lam + dele + 'n['+str(self.getColumnIndex(n)) + ']'
            dele = ','
        lam = lam + ')'
        return lam
    
    def getColumnHash(self,columnIndexs):
        #if self.hash.has_key(columnIndexs):
        #    return self.hash[columnIndexs]
            
        pivotkeys = {}

        if type(columnIndexs) == type('abc'):
            c = self.getColumnIndex(columnIndexs)
            for i,n in enumerate(self['data']):
                key = n[c]
                if pivotkeys.has_key(key):
                    pivotkeys[key].append(i)
                else:
                    pivotkeys[key] = [i,]
        else:
                
            for i,n in enumerate(self['data']):
                key = tuple([n[c] for c in columnIndexs])
                if pivotkeys.has_key(key):
                    pivotkeys[key].append(i)
                else:
                    pivotkeys[key] = [i,]
        
        #self.hash[columnIndexs] = pivotkeys
        
        return pivotkeys
    
    def sort(self,keyOrder,descending=False):
        lam = self.__getSortKeyComplex(keyOrder)
        self['data'].sort(key=lambda n: eval(lam),reverse=descending)
            
    def applyFilter(self,filter):
        
        if type(filter)==type('a'):
            filter = dataFilter(filter)
        
        newdata = []
        for n in filter.filter(self):
            newdata.append(n)
        self['data'] = newdata
        #print cc

    def deleteRows(self,startIndex,endIndex=None):
        """ Deletes rows starting from startIndex to endIndex. If endIndex is not provided, it defaults 
            to the last row in the table.
            Example: deleteRows(5)    <- deletes all the rows start at row 5
                     deleteRows(5,10) <- deletes rows 5 through 10
        """ 
        if type(startIndex) != type(1):
            raise TypeError
        
        if endIndex:
            if type(endIndex) != type(1):
                raise TypeError
            del self.data[startIndex:endIndex]
        else:
            del self.data[startIndex:]
            

    def filter(self,f):

        if type(f)==type('a'):
            f = dataFilter(f)

        
        ret= DataTable()
        ret['columns'] = self['columns'][:]
        newdata = []
        for n in f.filter(self):
            newdata.append(n)
        ret['data'] = newdata
        return ret
    
    def uniquify(self,columnNames):
        if type(columnNames) == type(''):
            columnNames = [columnNames,]
        cindex= self.getColumnIndex(columnNames)
        ret = DataTable()
        ret['columns'] = self['columns']
        ret['data'] = []
        
        chash = self.getColumnHash(cindex)
        for k,v in chash.items():
            ret['data'].append(self['data'][v[0]])
        return ret
    
    def __isCellEmpty(self,row,colnum):
        if row[colnum] == '' or row[colnum]==None:
            return True
        else:
            return False
    
    def projectionSummary(self,xcolumns,xtuples,ycolumns,ytuples,datacolumn):
        ret = DataTable()
        ret['columns'] = ['X']
        ret['data'] = []
        if xtuples == None:
            xtuples = self.getUniqueColumnValues(xcolumns)
        if ytuples == None:
            ytuples = self.getUniqueColumnValues(ycolumns)
        
        xc = self.getColumnIndex(xcolumns)
        yc = self.getColumnIndex(ycolumns)
        dc = self.getColumnIndex(datacolumn)
        
        nc = len(xtuples)
        nr = len(ytuples)
        numplots = nc * nr
        assert numplots <= 25
        
        chash = self.getColumnHash(xc + yc)
        print xtuples
        print ytuples
        
        rowaxes = {}
        colaxes = {}
        for c,x in enumerate(sorted(xtuples)):
            ret['columns'].append(":".join(x))
            
        for r,y in enumerate(ytuples):
            newrow = [":".join(y)]
            for c,x in enumerate(sorted(xtuples)):
                
                
                #fignum = r * nc + c + 1
                #print r,c,nr,nc,fignum
                key = tuple(x+y)
                if chash.has_key(key):
                    rowlist = chash[key]
                    if len(rowlist) > 1:
                        
                        raise "More than 1 data set found for key ",key
                    newrow.append(self.getData(rowlist[0],dc))
                    #ddata = self.dataTable.getData(rowlist[0],dc)
                else:
                    newrow.append(None)
                    #raise "Cannot find data for "+str(x)+" "+str(y)
                
            ret['data'].append(newrow)
        return ret
    

            
    
    def group(self,groupColumns,aggrigateSpec):
        return self.groupSummary(groupColumns, aggrigateSpec, False)
    
    def groupSummary(self,groupColumns,aggrigateSpec,groupSum=True):
        
        if type(groupColumns) == type(''):
            groupColumns1 = groupColumns.split(',')
            groupColumns = [x.strip() for x in groupColumns1]

        ret = DataTable()
        ret['columns'] = []
        ret['columns'].extend(groupColumns)
        
        columns = groupColumns[:]
        tup2sum = {}
        droppedRows = {}
        tupexprset = []
        if groupSum:
            for j in xrange(0,len(columns)):
                tupexpr = '('
                for n in columns[0:j+1]:
                    tupexpr += 'n[' + str(self.getColumnIndex(n)) + '],'
                tupexpr += ')'
                #print tupexpr
                tupexprset.append(tupexpr)
        else:
            tupexpr = '('
            for n in columns:
                tupexpr += 'n[' + str(self.getColumnIndex(n)) + '],'
            tupexpr += ')'
            tupexprset.append(tupexpr)
        
        #aggrigateSpec += ',count:'
        aggFunctionSplit = aggrigateSpec.split(',')
        aggFunctions = []
        for n in aggFunctionSplit:
            if n == '':
                continue
            ret['columns'].append(n)
            (func,cname) = n.split(':')
            droppedRows[func+':'+cname] = 0
            if cname == '':
                aggFunctions.append([func,None,None])
            else:
                aggFunctions.append( [func,cname,self.getColumnIndex(cname)] )
            columns.append(n)
    
        for n in self['data']:
            for t in tupexprset:
                newtup = eval(t)
                if tup2sum.has_key(newtup):
                    
                    newrow=tup2sum[newtup]
                    #print "cursum:",newrow,"n:",n
                    tcol=0
                    for func,cname,cindex in aggFunctions:
                        #print n,cindex
                        if cindex:
                            if self.__isCellEmpty(n,cindex):
                                droppedRows[func+':'+cname] = droppedRows[func+':'+cname] +1
                                continue
                        if func == 'min':
                            if n[cindex] < newrow[tcol]:
                                newrow[tcol] = n[cindex]
                
                        if func == 'max':
                            if n[cindex] > newrow[tcol]:
                                newrow[tcol] = n[cindex]

                        if func == 'mean':
                            x = (x[0]+n[cindex],x[1]+1)
                            newrow[tcol] = newrow[tcol] + n[cindex]
                    
                        if func == 'sum':
                            newrow[tcol] = newrow[tcol] + n[cindex]
                    
                        if func == 'count' or func=='collect':
                            if n[cindex] != None:
                                newrow[tcol].add(n[cindex])# = '' # = newrow[tcol] + 1
                        
                        if func == 'median':
                            if n[cindex] != None:
                                newrow[tcol].append(n[cindex])
                        tcol = tcol + 1

                else:
                    sumcolumns = []
                    for func,cname,cindex in aggFunctions:
                        if func == 'max' or func == 'min' or func == 'mean' or func=='sum':
                #print cindex,rows[0]
                            sumcolumns.append(n[cindex])
                        if func == 'mean':
                            x = (0.0,0)
                            if n[cindex]!=None:
                                x = (x[0]+n[cindex],1)
                        
                        if func == 'count' or func == 'collect':
                            x = Set()
                            if n[cindex] != None:
                                x.add(n[cindex])
                            sumcolumns.append(x)
                        if func == 'median':
                            x = []
                            if n[cindex] != None:
                                x.append(n[cindex])
                            sumcolumns.append(x)
                    tup2sum[newtup] = sumcolumns
        
        
        
                    
               
        
        ret['data'] = []
        for k,val in sorted(tup2sum.items()):
            r = []
            r.extend(k)
            if len(k) < len(groupColumns):
                temp = [None for n in range(len(groupColumns)-len(k))]
                r.extend(temp)
            tcol = 0
            for func,cname,cindex in aggFunctions:
                if func == 'max' or func == 'min' or func=='sum':
                    pass
                elif func=='count':
                        #print aggFunctions
                        #print val
                    val[tcol] = len(val[tcol])
                elif func=='mean':
                    val[tcol] = float(val[tcol][0])/float(val[tcol[1]])
                elif func == 'median':
                    val[tcol] = self.findMedian(val[tcol])
                    
                tcol = tcol+ 1    
            r.extend(val)
            ret['data'].append(r)
        return ret    
        #print tup2sum
        #print droppedRows
        
    

        
    def __getTupleConstructionString(self,columnIndex,row='n'):
        st = '('
        for i in columnIndex:
            st = st + row + '[' + str(i) + '],'
        st = st + ')'
        return st
    


    def getUniqueColumnValuesIncludingNone(self,columnNames):
        vals = Set()
        #print "ColumnnNames:",columnNames,len(columnNames)
        #if columnNames=='':
            
        #    return vals
        if type(columnNames) == type(''):
            columnNames = columnNames.split(',')
        gi = self.getColumnIndex(columnNames)
        st = self.__getTupleConstructionString(gi)
        
        for n in self.dataMask.filter(self):
            tup = eval(st)
            #if None in tup:
            #    pass
            #else:
            vals.add(eval(st))
            #vals.add(n[gi])
        return vals

    def getUniqueColumnValuesSingleColumn(self,columnName):
        assert type(columnName) == type('')
        vals = Set()
        gi = self.getColumnIndex(columnName)
        #st = self.__getTupleConstructionString(gi)
        
        for n in self.dataMask.filter(self):
            vals.add(n[gi])

        return vals    
    
    def getUniqueColumnValues(self,columnNames):
        if type(columnNames) == type(''):
            columnNames = columnNames.split(',')
        vals = Set()
        gi = self.getColumnIndex(columnNames)
        st = self.__getTupleConstructionString(gi)
        
        for n in self.dataMask.filter(self):
            tup = eval(st)
            if None in tup:
                pass
            else:
                vals.add(eval(st))
            #vals.add(n[gi])
        return vals

    def setColumnValue(self,columnName,value,dataFilter):
        """ Sets the value of 'columnName' for all the rows selected by the dataFilter.
            Example: setColumnValue('failures',0,dataFilter('wafer == 19'))
        """
        columnIndex= self.getColumnIndex(columnName)
        for n in dataFilter.filter(self):
            n[columnIndex] = value
        

    def searchAndReplace(self,searchFor,replaceWith):
        """ Performs search and replace on all the data in the table.
        Example: searchAndReplace('invalid',0.200) <- replaces all 'invalid' with 0.200
                 searchAndReplace('PDB_','GAGA_')  <- replaces all substrings PDB_ with GAGA_
                 
        """
        if type(searchFor) == type('') and type(replaceWith) == type(''):
            s = True
        else:
            s = False
        for c in self.data:
            for i,n in enumerate(c):
                try:
                    if s:
                        if type(n) == type(''):
                            c[i] = n.replace(searchFor,replaceWith)
                    else:
                        if n == searchFor:
                            c[i]=replaceWith
                except:
                    print "Error in data:",c
                    raise
                    
                    
    def groupSummaryAdvanced(self,groupColumns,groupSum=False,**aggregates):
        return self.groupSummaryAdvancedInternal(groupColumns, groupSum,aggregates)
        
                            
    def groupSummaryAdvancedInternal(self,groupColumns,groupSum,aggregates):
        ''' Example: table.groupSummaryAdvanced('col1,col2',sum='sum(col3),col1 == "P"')
        '''
        
        if type(groupColumns) == type(''):
            groupColumns1 = groupColumns.split(',')
            groupColumns = [x.strip() for x in groupColumns1]

    

        ret = DataTable()
        ret['columns'] = []
        ret['columns'].extend(groupColumns)
        
        columns = groupColumns[:]
        tup2sum = {}
        droppedRows = {}
        tupexprset = []
        if groupSum:
            for j in xrange(0,len(columns)):
                tupexpr = '('
                for n in columns[0:j+1]:
                    tupexpr += 'n[' + str(self.getColumnIndex(n)) + '],'
                tupexpr += ')'
                #print tupexpr
                tupexprset.append(tupexpr)
        else:
            tupexpr = '('
            for n in columns:
                tupexpr += 'n[' + str(self.getColumnIndex(n)) + '],'
            tupexpr += ')'
            tupexprset.append(tupexpr)
        
        #print "SummaryDebug................."
        #for n in tupexprset:
        #    print n
        
        #aggrigateSpec += ',count:'
        #aggFunctionSplit = aggrigateSpec.split(',')
        aggFunctions = []
        for k,v in aggregates.items():
#        for n in aggFunctionSplit:
            #if n == '':
            #    continue
            ret['columns'].append(k)
            v.strip(' \t')
            vsp = v.split(',')
            exp = vsp[0]
            cond = 'True'
            if len(vsp) == 2:
                cond = vsp[1]
            else:
                if len(vsp) > 2:
                    print "Error: Invalid aggregate spec!!"
                    raise
            #cond = compile(cond,'<string>','eval')
            (func,cname) = exp.split(':')
            droppedRows[func+':'+cname] = 0
            if cname == '':
                aggFunctions.append([func,None,None,cond])
            else:
                aggFunctions.append( [func,cname,self.getColumnIndex(cname),cond] )
            columns.append(n)
    
        #print "aggFunctions..."
        #for n in aggFunctions:
        #    print n
    
        for nindex,n in enumerate(self['data']):
            rowdict=self.getItem(nindex)
            for t in tupexprset:
                newtup = eval(t)
                if tup2sum.has_key(newtup):
                    pass
                else:
                    sumcolumns = []
                    for func,cname,cindex,cond in aggFunctions:
                        if func == 'max' or func =='min':
                            sumcolumns.append(None)
                        if func =='sum':
                            sumcolumns.append(0)
                            
                        if func == 'mean':
                            sumcolumns.append([0,0])
                        
                        if func == 'count' or func == 'collect' or func == 'countKeys' or func == 'collectKeys':
                            x = Set()
                            sumcolumns.append(x)
                        
                        if func == 'collectKeysWithValueSum':
                            x = {}
                            sumcolumns.append(x)
                        
                        
                        if func == 'collectKeysWithValueMean':
                            x = {}
                            sumcolumns.append(x)
                        
                        if func == 'collectKeysWithValues':
                            x={}
                            sumcolumns.append(x)
                        
                    
                        if func == 'collectall' or func == 'collectall2absValue' or func == 'median':
                            x = []
                            sumcolumns.append(x)

                            
                    tup2sum[newtup] = sumcolumns
                    
                    #print "newtup:",newtup,tup2sum[newtup]
                    
                newrow=tup2sum[newtup]
                #print "cursum:",newrow,"n:",n

                tcol=0
                for func,cname,cindex,cond in aggFunctions:
                    #print func,cname,cindex,cond
                    if eval(cond,rowdict):
                        pass
                    else:
                        tcol = tcol + 1
                        continue
                        
                        #check if row passes condition
                        
                    #print n,cindex
                    if cindex:
                        if self.__isCellEmpty(n,cindex):
                            droppedRows[func+':'+cname] = droppedRows[func+':'+cname] +1
                            tcol = tcol + 1
                            continue
                        
                    if func == 'min':
                        if newrow[tcol] == None:
                            newrow[tcol] = n[cindex]
                        else:
                            if n[cindex] < newrow[tcol]:
                                newrow[tcol] = n[cindex]
            
                    elif func == 'max':
                        if newrow[tcol] == None:
                            newrow[tcol] = n[cindex]
                        else:
                            if n[cindex] > newrow[tcol]:
                                newrow[tcol] = n[cindex]

                    elif func == 'mean':
                        #x = (x[0]+n[cindex],x[1]+1)
                        newrow[tcol][0] = newrow[tcol][0] + n[cindex]
                        newrow[tcol][1] = newrow[tcol][1] + 1
                
                    elif func == 'sum':
                        newrow[tcol] = newrow[tcol] + n[cindex]
                
                    elif func == 'count' or func=='collect':
                        if n[cindex] != None:
                            newrow[tcol].add(n[cindex])# = '' # = newrow[tcol] + 1
                
                    elif func == 'countKeys' or func =='collectKeys':
                        for kk in n[cindex].keys():
                            newrow[tcol].add(kk)
                    elif func == 'collectKeysWithValueSum':
                        for kk in n[cindex].keys():
                            try:
                                newrow[tcol][kk] = newrow[tcol][kk] + n[cindex][kk]
                            except KeyError:
                                newrow[tcol][kk] = n[cindex][kk]

                    elif func == 'collectKeysWithValueMean' or func=='collectKeysWithValues':
                        for kk in n[cindex].keys():
                            try:
                                if type(n[cindex][kk])==type([]):
                                    newrow[tcol][kk].extend(n[cindex][kk])
                                else:
                                    newrow[tcol][kk].append(n[cindex][kk])
                            except KeyError:
                                if type(n[cindex][kk])==type([]):
                                    newrow[tcol][kk] = []
                                    newrow[tcol][kk].extend(n[cindex][kk])
                                else:
                                    newrow[tcol][kk] = [n[cindex][kk],]
                    
                
                    elif func == 'collectall' or func == 'median':
                        newrow[tcol].append(n[cindex])
                                                    
                    elif func == 'collectall2absValue' :
                        newrow[tcol].append(math.fabs(n[cindex]))

                    else:
                        print "Error: Invalid aggregate function:",func
                        raise ValueError
                    
                    tcol = tcol + 1

         

        
        
        
                    
               
        
        ret['data'] = []
        for k,val in sorted(tup2sum.items()):
            r = []
            r.extend(k)
            if len(k) < len(groupColumns):
                temp = [None for n in range(len(groupColumns)-len(k))]
                r.extend(temp)
            tcol = 0
            for func,cname,cindex,cond in aggFunctions:
                if func == 'max' or func == 'min' or func=='sum':
                    pass
                elif func=='count' or func == 'countKeys':
                        #print aggFunctions
                        #print val
                    val[tcol] = len(val[tcol])
                
                elif func=='mean':
                    try:
                        val[tcol] = float(val[tcol][0])/float(val[tcol][1])
                    except ZeroDivisionError:
                        if val[tcol][0] == 0:
                            val[tcol] = 0
                        else:
                            assert 1==0
                elif func=='collectKeysWithValueMean':
                    for k in val[tcol].keys():
                        v = val[tcol][k]
                        if len(v)==0:
                            val[tcol][k]=0
                        else:
                            val[tcol][k]=sum(v,0.0)/len(v)
                elif func == 'median':
                    val[tcol] = self.findMedian(val[tcol])
                        
                tcol = tcol+ 1    
            r.extend(val)
            ret['data'].append(r)
        return ret    
        #print tup2sum
        #print droppedRows


    def getColumnVectors(self):
        ret = {}
        for i,n in enumerate(self['columns']):
            l= []
            for d in self['data']:
                l.append(d[i])
            ret[n] = l
        return ret



    def convertRowsToColumns(self,groupColumns,categoryColumns,valueColumns):

        if type(categoryColumns)==type(''):
            categoryColumns=categoryColumns.split(',')
        if type(valueColumns)==type(''):
            valueColumns=valueColumns.split(',')

        if type(groupColumns)==type(''):
            groupColumns=groupColumns.split(',')
        
        #groupColumns=[]    
        #for n in self['columns']:
        #    if (n in categoryColumns) or (n in valueColumns):
        #        pass
        #    else:
        #        groupColumns.append(n.strip())
        
        #if type(groupColumns) == type(''):
        #    groupColumns1 = groupColumns.split(',')
        #groupColumns = [x.strip() for x in groupColumns1]

    

        ret = DataTable()
        ret['columns'] = []
        ret['columns'].extend(groupColumns)
        cats=self.getUniqueColumnValues(categoryColumns)
        vcindex = self.getColumnIndex(valueColumns)
        ccindex= self.getColumnIndex(categoryColumns)
        blankcol=[]
        name2col={}
        coli=0
        for n in valueColumns:
            for m in sorted(cats):
                
                ret['columns'].append(n+'_'+'_'.join(m))
                name2col[n+'_'+'_'.join(m)] = coli
                coli+=1
                blankcol.append(None)
                
                
                
        columns = groupColumns[:]
        tup2sum = {}
        droppedRows = {}
        #tupexprset = []
        tupexpr = '('
        for n in columns:
            tupexpr += 'n[' + str(self.getColumnIndex(n)) + '],'
        tupexpr += ')'
        #tupexprset.append(tupexpr)
        
        for nindex,n in enumerate(self['data']):
            rowdict=self.getItem(nindex)
            t = tupexpr
            if 1:
            #for t in tupexprset:
                newtup = eval(t)
                if tup2sum.has_key(newtup):
                    pass
                else:
                    tup2sum[newtup] = blankcol[:]
                    
                    #print "newtup:",newtup,tup2sum[newtup]
                    
                newrow=tup2sum[newtup]
                #print "cursum:",newrow,"n:",n
                for vci,vc in enumerate(valueColumns):
                    
                    dcats=[]
                    tocol = vc + '_' + '_'.join([n[cc] for cc in ccindex])
                    newrow[name2col[tocol]]=n[vcindex[vci]]
        
        ret['data'] = []
        for k,val in sorted(tup2sum.items()):
            r = []
            r.extend(k)
            r.extend(val)
            ret['data'].append(r)
        return ret    
    

#    def convertRowsToColumns(self,categoryColumns):
#        #if len(self.indexColumns) == 0:
#        #    print "Must define indexColumns before using this API: convertCategoryToColumns"
#        
#        ret= DataTable()
#        
#        g = self.getUniqueColumnValues(categoryColumns)
#        
#        newcols = []
#        for n in self['columns']:
#            if self.isIndexColumn(n) >= 0:
#                if n != categoryColumnName:
#                    newcols.append(n)
#        for n in g:
            
        
        

